package com.attilax.spider;

import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeoutException;
import java.util.function.Function;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.attilax.dataspider.HttpDownloader;
import com.attilax.exception.ExUtil;
import com.attilax.io.filex;
import com.attilax.net.urlUtil;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

public class SpiderFrmwkPubPart {
	public Function<String, Boolean> titleChker;
	public Function<Map, Boolean> clawFinishHandler;
	public Map deepFilter = Maps.newConcurrentMap();
	public String seedUrl;
	public String savedir;

	public static void main(String[] args) {

	}

	public List<String> getUrlsByHtml(String html) {
		List<String> li = Lists.newArrayList();
		Document doc = null;
		doc = Jsoup.parse(html);
		String title = doc.title();
		Elements as = doc.getElementsByTag("a");
		for (Element e : as) {
			String href = e.attr("href");

			if (urlUtil.isFullUrl(href))
				href = href;
			else if (urlUtil.isAbsUrl(href)) {
				href = urlUtil.getUrlHead(seedUrl) + href;
			} else if (urlUtil.isRltUrl(href)) {
				href = urlUtil.addLastSplash(seedUrl) + href;
			}

			String title2 = e.text();
			System.out.println(title2);
			try {
				this.titleChker.apply(title);
			} catch (TitNotOk e1) {
				System.out.println("--tie too shortL:" + title);
				continue;
			}
			// next.add(href);
			li.add(href);
		}
		return li;
	}

	public void clawFinishHandlerDefalt_apply(Map m) {
		String fname = filex.fileNameEncode(m.get("title").toString()) + ".html";
		String fileName = savedir+"\\" + fname;
		if (!new java.io.File(fileName).exists())
		{
			String picurl=(String) m.get("url");
			try {
				new HttpDownloader().down(picurl, fileName, 90);
			} catch (TimeoutException e) {
				ExUtil.throwExV2(e);
			}
		
			//filex.save(m.get("content").toString(), fileName, "gbk");
			
		}
	//	return null;
	}

}
